package main

import (
	"errors"
	"github.com/PuerkitoBio/goquery"
	"log"
	"net/http"
	netUrl "net/url"
	"strconv"
	"strings"
	"time"
)

var skipPath = map[string]int8{
	"con": 1,
	"prn": 1,
	"aux": 1,
	"nul": 1,
}

// 在籍演员名字检索
var keyList = []string{
	"a", "e", "i", "o", "u",
	"ka", "ke", "ki", "ko", "ku",
	"sa", "se", "si", "so", "su",
	"ta", "te", "ti", "to", "tu",
	"na", "ne", "ni", "no", "nu",
	"ha", "he", "hi", "ho", "hu",
	"ma", "me", "mi", "mo", "mu",
	"ya", "yo", "yu",
	"ra", "re", "ri", "ro", "ru",
	"wa",
}

// 素人演员名字检索
var suKeyList = []string{
	"a", "e", "i", "o", "u",
	"ka", "ke", "ki", "ko", "ku",
	"sa", "se", "si", "so", "su",
	"ta", "te", "ti", "to", "tu",
	"na", "ne", "ni", "no", "nu",
	"ha", "he", "hi", "ho", "hu",
	"ma", "me", "mi", "mo", "mu",
	"ya", "yo", "yu",
	"ra", "re", "ri", "ro", "ru",
	"wa", "wo", "nn", "num",
}
var waitTime = 3 * time.Second
var httpClient = &http.Client{
	Timeout: 3 * waitTime,
}

// 获取所有在籍演员
func getActresses() {
	mDB.getActorQur()
	mDB.getActorUpt()
	mDB.getActorIns()
	mDB.getPicsIns()
	for _, k := range keyList {
		getActress(k)
	}
	log.Println("getActresses END")
}

// 获取单个在籍演员
func getActress(key string) {
	url := "https://www.dmm.co.jp/digital/videoa/-/actress/=/keyword=" + key + "/page=1/"
	doc, err := getPage(url)
	if err != nil {
		log.Println(err)
		return
	}
	// Find the review items
	ps := doc.Find(".list-boxcaptside p")
	if ps.Length() <= 1 {
		log.Println(".list-boxcaptside p")
		return
	}
	s := ps.Text()
	n, _ := strconv.Atoi(s[:strings.Index(s, "人中")])
	c := (n + 99) / 100
	log.Println(c)

	for i := 1; i <= c; i++ {
		time.Sleep(waitTime)
		url = "https://www.dmm.co.jp/digital/videoa/-/actress/=/keyword=" + key + "/page=" + strconv.Itoa(i) + "/"
		doc, err = getPage(url)
		if err != nil {
			log.Println(url, err)
			continue
		}

		doc.Find(".act-box-100 li").Each(func(i int, li *goquery.Selection) {
			go func() {
				actor := actress{}

				aTag := li.Find("a")
				href, _ := aTag.Attr("href")
				n := strings.Index(href, "id=")
				if n == -1 {
					return
				}
				s := href[n+3:]
				aid := s[:strings.Index(s, "/")]
				actor.aid, _ = strconv.Atoi(aid)

				spanTags := aTag.Find("span")
				actor.nameJP = spanTags.Eq(0).Text()

				tt := spanTags.Eq(1).Text()
				log.Println(tt)
				actor.movies, _ = strconv.Atoi(tt[strings.Index(tt, "：")+3:])

				if mDB.GetActressNameByID(actor.aid) != "" {
					log.Println("actor existed. Aid:", actor.aid)
					mDB.UptActressMovies(actor.aid, actor.movies)
					return
				}

				imgTag := aTag.Find("img")
				src, _ := imgTag.Attr("src")
				suffix := src[strings.LastIndex(src, "/"):]
				actor.name, _ = imgTag.Attr("alt")
				actor.avatar = "/dmm/actress" + suffix
				go addDownload("aid"+aid, "./dmm/actress/", src)

				mDB.InsertActress(&actor)

				log.Println(actor)
			}()
		})
	}
	log.Println("getActress END")
}

func getMissActress() {
	mDB.getActorQur()
	mDB.getActorUpt()
	mDB.getActorIns()

	url := "https://www.dmm.co.jp/digital/videoa/-/list/=/article=actress/id="

	miss := mDB.getMissActress()
	for _, act := range miss {
		doc, err := getPage(url + strconv.Itoa(act) + "/sort=ranking/view=text/")
		if err != nil {
			miss = append(miss, act)
			continue
		}

		actor := &actress{}
		text := doc.Find("title").Text()
		n := strings.Index(text, " ")
		if n == -1 {

		}
		text = text[:n]
		if len(text) < 1 {
			continue
		}
		n1 := strings.Index(text, "(")
		if n1 == -1 {
			actor.name = text
			actor.nameJP = text
		} else {
			n2 := strings.Index(text, ")")
			actor.name = text[:n1]
			actor.nameJP = text[n1+1 : n2]
		}
		actor.aid = act

		p := doc.Find(".list-capt .list-boxcaptside p").Text()
		pn := strings.Index(p, "タ")
		if pn > 0 {
			actor.movies, _ = strconv.Atoi(p[:pn])
		}

		mDB.InsertActress(actor)
	}
}

// ----------movies-----------//
func PreGetDailyMovies(page int) {
	url := "https://www.dmm.co.jp/digital/videoa/-/list/=/sort=date/page="

	for i := 1; i <= page; i++ {
		time.Sleep(waitTime)
		doc, err := getPage(url + strconv.Itoa(i))
		if err != nil {
			log.Println(url, err)
			continue
		}
		PreGetMoviesExce(doc)
	}
	log.Println("PreGetDailyMovies END")
}

func PreGetActorMovies() {
	list := mDB.GetScanActor(1400)
	if len(list) <= 0 {
		log.Println("PreGetActorMovies END")
		return
	}

	for _, act := range list {
		PreGetOneActorMovies(&act)
	}
	//PreGetActorMovies()
}

func PreGetOneActorMovies(act *actress) {
	url := "https://www.dmm.co.jp/digital/videoa/-/list/=/article=actress/id="
	tmpUrl := url + strconv.Itoa(act.aid)

	// 获取影片数并更新
	doc, err := getPage(tmpUrl)
	if err != nil {
		if Status404 == err {
			mDB.UptActressStatus(act.aid)
		}
		log.Println(err)
		return
	}
	p := doc.Find(".list-capt .list-boxcaptside p").Text()
	pn := strings.Index(p, "タ")
	if pn > 0 {
		movies, _ := strconv.Atoi(p[:pn])
		if movies != act.movies && movies > 0 {
			act.movies = movies
			mDB.UptActressMovies(act.aid, movies)
		}
	}

	page := (act.movies + 119) / 120

	PreGetMoviesExce(doc)
	if page > 1 {
		for i := 2; i <= page; i++ {
			time.Sleep(waitTime)
			doc, err := getPage(tmpUrl + "/page=" + strconv.Itoa(i))
			if err != nil {
				log.Println(tmpUrl, err)
				continue
			}
			PreGetMoviesExce(doc)
		}
	}

	mDB.UptActressScan(act.aid, time.Now().Format("2006-01-02 15:04:05"))
}

func PreGetMoviesExce(doc *goquery.Document) {
	//log.Println(doc.Text())
	doc.Find(".tmb a").Each(func(i int, li *goquery.Selection) {
		go func() {
			href, _ := li.Attr("href")
			n := strings.Index(href, "cid=")
			if n == -1 {
				return
			}

			s := href[n+4:]
			cid := s[:strings.Index(s, "/")]

			if mDB.IsMovieExist(cid) {
				log.Println("movies existed ", cid)
				return
			}

			mDB.PreInsertMovie(cid)
			log.Println("Insert movie ", cid)
		}()
	})
}

func GetAllMoviesDetail() bool {
	list := mDB.GetScanMovies(300)
	if len(list) <= 0 {
		log.Println("GetAllMoviesDetail END")
		return false
	}

	// 先初始化 stmt，避免资源冲突
	mDB.getMA()
	mDB.getMG()
	mDB.getDirectorIns()
	mDB.getSeriesIns()
	mDB.getLabelIns()
	mDB.getPicsIns()
	mDB.getUpdateMovieStmt()

	for _, cid := range list {
		//if len(cid) > 4 {
		//	if _, r := strconv.Atoi(cid[len(cid)-4:len(cid)-3]); r != nil {
		//		mDB.UptMovieFilled(4, cid)
		//		log.Println("cid is short ---")
		//		continue
		//	}
		//}
		getMovieCh <- 1
		go GetMovieDetail(cid)
		hour := time.Now().Hour()
		if hour >= 20 || hour < 8 {
			time.Sleep(waitTime)
		} else {
			time.Sleep(waitTime)
		}
	}
	return true
}

func GetMovieDetail(cid string) {
	defer func() { <-getMovieCh }()

	url := "https://www.dmm.co.jp/digital/videoa/-/detail/=/cid="
	path := "./dmm/movies/" + cid[:3] + "/" + cid + "/"
	if _, exist := skipPath[cid[:3]]; exist == true {
		path = "./dmm/movies/" + cid[:4] + "/" + cid + "/"
	}

	doc, err := getPage(url + cid)
	if err != nil {
		e4 := strings.Index(err.Error(), "404")
		if e4 != -1 {
			mDB.UptMovieFilled(2, cid)
		}
		log.Println(cid, err)
		return
	} else {
		log.Println("getPage Back", cid)
	}
	mv := &movie{
		cid:    cid,
		filled: 1,
	}

	mv.title, _ = doc.Find("#sample-video img").Attr("alt")
	mv.ps, _ = doc.Find("#sample-video img").Attr("src")
	mv.pl = strings.Replace(mv.ps, "ps.jpg", "pl.jpg", -1)
	mv.sample = doc.Find("#detail-sample-movie").Length()
	mv.fcid = mv.ps[strings.LastIndex(mv.ps, "/")+1 : len(mv.ps)-6]

	table := doc.Find("#mu table table tr")
	mv.saleAt = strings.TrimSpace(table.Eq(2).Find("td").Last().Text())
	mt, _ := time.Parse("2006/01/02", mv.saleAt[:10])
	mv.saleAt = mt.Format("2006-01-02 15:04:05")
	mv.soldAt = mt.Unix()

	n := -1
	duration := strings.TrimSpace(table.Eq(4).Find("td").Last().Text())
	n = strings.Index(duration, "分")
	if n == -1 {
		mDB.UptMovieFilled(3, cid)
		return
	}
	mv.duration, _ = strconv.Atoi(duration[:n])

	if act := table.Eq(5).Find("a"); act.Length() > 0 {
		act.Each(func(i int, selection *goquery.Selection) {
			s, e := selection.Attr("href")
			if !e {
				return
			}
			if n = strings.Index(s, "id="); n != -1 {
				aid := s[n+3 : len(s)-1]
				aid8, _ := strconv.Atoi(aid)
				go mDB.InsertMA(aid8, cid)
			}
		})
	} else {
		mv.su = "----"
	}
	if dir := table.Eq(6).Find("a"); dir.Length() > 0 {
		s, e := dir.First().Attr("href")
		if !e {
			return
		}
		if n = strings.Index(s, "id="); n != -1 {
			did := s[n+3 : len(s)-1]
			mv.directorId, _ = strconv.Atoi(did)

			if !mDB.IsDirectorExist(mv.directorId) {
				dirName := dir.First().Text()
				mDB.AddDirector(mv.directorId, dirName)
			}
		}
	}
	if ser := table.Eq(7).Find("a"); ser.Length() > 0 {
		s, e := ser.First().Attr("href")
		if !e {
			return
		}
		if n = strings.Index(s, "id="); n != -1 {
			sid := s[n+3 : len(s)-1]
			mv.seriesId, _ = strconv.Atoi(sid)

			if !mDB.IsSeriesExist(mv.seriesId) {
				seriesName := ser.First().Text()
				mDB.InsertSeries(mv.seriesId, seriesName)
			}
		}
	}
	if mak := table.Eq(8).Find("a"); mak.Length() > 0 {
		s, e := mak.First().Attr("href")
		if !e {
			return
		}
		if n = strings.Index(s, "id="); n != -1 {
			mid := s[n+3 : len(s)-1]
			mv.makerId, _ = strconv.Atoi(mid)
		}
	}
	if lab := table.Eq(9).Find("a"); lab.Length() > 0 {
		s, e := lab.First().Attr("href")
		if !e {
			return
		}
		if n = strings.Index(s, "id="); n != -1 {
			lid := s[n+3 : len(s)-1]
			mv.labelId, _ = strconv.Atoi(lid)

			if !mDB.IsLabelExist(mv.labelId) {
				labelName := lab.First().Text()
				mDB.AddLabel(mv.labelId, labelName)
			}
		}
	}
	if gen := table.Eq(10).Find("a"); gen.Length() > 0 {
		gen.Each(func(i int, selection *goquery.Selection) {
			s, e := selection.Attr("href")
			if !e {
				return
			}
			if n = strings.Index(s, "id="); n != -1 {
				gid := s[n+3 : len(s)-1]
				gid8, _ := strconv.Atoi(gid)
				go mDB.InsertMG(gid8, cid)
			}
		})
	}

	intr, _ := doc.Find(".mg-b20.lh4").Html()
	intr = strings.Replace(intr, "<br/>", "", -1)
	n = strings.Index(intr, "<")
	if n > -1 {
		intr = intr[:strings.Index(intr, "<")]
	}
	mv.introduce = strings.TrimSpace(intr)

	pics := doc.Find("#sample-image-block img")
	mv.pics = pics.Length()
	pics.Each(func(i int, li *goquery.Selection) {
		src, _ := li.Attr("src")
		if n = strings.Index(src, "-"); n != -1 {
			src = src[:n] + "jp" + src[n:]
			go addDownload(cid+"-"+strconv.Itoa(i), path, src)
		}
	})

	go addDownload(cid+"ps", path, mv.ps)
	go addDownload(cid+"pl", path, mv.pl)

	mDB.UpdateMovie(mv)
	log.Println("update movie", cid)
}

func FixMoviesActress() {
	//1stars00199
	for i := 329337; i > 0; {
		ids, last := mDB.getFailedMA(i)
		for _, cid := range ids {
			FixMoviesActor(cid)
			time.Sleep(100 * time.Millisecond)
		}
		i = last
		log.Println(last)
	}
}

func FixMoviesActor(cid string) {
	url := "https://www.dmm.co.jp/digital/videoa/-/detail/=/cid="

	doc, err := getPage(url + cid)
	if err != nil {
		return
	}

	table := doc.Find("#mu table table tr")
	if act := table.Eq(5).Find("a"); act.Length() > 0 {
		act.Each(func(i int, selection *goquery.Selection) {
			s, e := selection.Attr("href")
			if !e {
				return
			}
			if n := strings.Index(s, "id="); n != -1 {
				aid := s[n+3 : len(s)-1]
				aid8, _ := strconv.Atoi(aid)
				log.Println(aid, "----", cid)
				go mDB.InsertMA(aid8, cid)
			}
		})
	}
}

//--------------Genre------------//

func getGenre() {
	url := "https://www.dmm.co.jp/digital/videoa/-/genre/=/display=syllabary/"
	surl := "https://www.dmm.co.jp/digital/videoc/-/genre/=/display=syllabary/"

	if doc, err := getPage(url); err == nil {
		explainGenre(doc)
	}
	if doc, err := getPage(surl); err == nil {
		explainGenre(doc)
	}
	log.Println("getGenre END")
}

func getGenres() {
	for i := 593; i < 8600; i++ {
		if getOneGenre(i) {
			time.Sleep(waitTime)
		}
	}
}

func getOneGenre(gid int) bool {
	gidx := strconv.Itoa(gid)
	if genre := mDB.GetGenreByID(gid); genre != "" {
		log.Println(genre, "existed")
		return false
	}

	// --
	url := "https://www.dmm.co.jp/digital/videoa/-/list/=/article=keyword/id=" + gidx
	if doc, err := getPage(url); err == nil {
		genre := doc.Find(".active span").Text()
		mDB.InsertGenre(gid, genre)
		return true
	}

	// su
	url = "https://www.dmm.co.jp/digital/videoc/-/list/=/article=keyword/id=" + gidx
	if doc, err := getPage(url); err == nil {
		genre := doc.Find(".active span").Text()
		mDB.InsertGenre(gid, genre)
		return true
	}

	return true
}

func explainGenre(doc *goquery.Document) {
	doc.Find(".d-boxtbllist a").Each(func(i int, li *goquery.Selection) {
		if li.Find("img").Length() > 0 {
			return
		}

		genre := li.Text()
		txt, e := li.Attr("href")
		gid := ""
		if e {
			gid = txt[strings.Index(txt, "id=")+3 : len(txt)-1]
		}
		log.Println(genre, gid, txt)
		gid8, _ := strconv.Atoi(gid)

		if mDB.GetGenreByID(gid8) != "" {
			log.Println(genre, "existed")
			return
		}

		mDB.InsertGenre(gid8, genre)
	})
}

//--------------maker------------//

func getMakers() {
	// 若发现新制片，则全部查询一遍
	for _, k := range suKeyList {
		getMaker(k)
		time.Sleep(waitTime)
	}
	log.Println("getMakers END")
}

func getNewMaker() bool {
	hadNew := false
	url := "https://www.dmm.co.jp/digital/videoa/-/maker/"
	doc, err := getPage(url)
	if err != nil {
		log.Println(err)
		return hadNew
	}

	doc.Find(".d-sect").Eq(1).Find(".d-item").Each(func(i int, li *goquery.Selection) {
		if getMakerExc(li) {
			hadNew = true
		}
	})
	return hadNew
}

func getMaker(symbol string) bool {
	url := "https://www.dmm.co.jp/digital/videoa/-/maker/=/keyword=" + symbol
	doc, err := getPage(url)
	if err != nil {
		log.Println(err)
		return false
	}

	doc.Find(".d-unit").Each(func(i int, li *goquery.Selection) {
		go getMakerExc(li)
	})
	return false
}
func getMakerExc(li *goquery.Selection) bool {
	a := li.Find("a")
	if a.Length() <= 0 {
		log.Println("No Maker In")
		return false
	}

	txt, _ := a.Attr("href")
	mid := txt[strings.Index(txt, "id=")+3 : len(txt)-1]
	mid8, _ := strconv.Atoi(mid)
	if mDB.IsMakerExist(mid8) {
		log.Println(mid8, "existed")
		return false
	}

	img, _ := li.Find("img").Attr("src")
	logo := img[strings.LastIndex(img, "/")+1:]
	go addDownload("mid"+mid, "./dmm/maker_logo/", img)

	maker := li.Find(".d-ttllarge").Text()
	slogan := li.Find(".d-datacol p").Text()

	mDB.InsertMaker(mid8, maker, "/dmm/maker_logo/"+logo, slogan)

	log.Println(mid8, maker)
	return true
}

func PreGetMakerMovies() {
	list := mDB.GetScanMaker(40)
	if len(list) <= 0 {
		log.Println("PreGetMakerMovies END")
		return
	}
	url := "https://www.dmm.co.jp/digital/videoa/-/list/=/article=maker/id="

	for _, mid := range list {
		tmpUrl := url + strconv.Itoa(mid)
		doc, err := getPage(tmpUrl)
		if Status404 == err {
			mDB.UptMakerStatus(mid)
		}
		if err != nil {
			log.Println(tmpUrl, err)
			continue
		}
		PreGetMoviesExce(doc)

		pageBox := doc.Find(".list-boxpagenation a")
		if pageBox.Length() <= 0 {
			log.Println("Only Page 1")
			mDB.UptMakerScan(mid, time.Now().Format("2006-01-02 15:04:05"))
			time.Sleep(waitTime)
			continue
		}

		var aTag *goquery.Selection
		txt := pageBox.Last().Text()
		if txt == "次へ" {
			aTag = pageBox.Eq(pageBox.Length() - 2)
		} else {
			aTag = pageBox.Last()
		}
		h, _ := aTag.Attr("href")
		index := strings.Index(h, "page=")
		if index == -1 {
			continue
		}
		page, _ := strconv.Atoi(h[index+5 : len(h)-1])
		log.Println(page)

		for i := 2; i <= page; i++ {
			time.Sleep(waitTime)
			doc, err := getPage(tmpUrl + "/page=" + strconv.Itoa(i))
			if err != nil {
				log.Println(tmpUrl, err)
				continue
			}
			PreGetMoviesExce(doc)
		}

		mDB.UptMakerScan(mid, time.Now().Format("2006-01-02 15:04:05"))
	}
	//PreGetMakerMovies()
}

//--------------Series------------//

func getSeries(page int) bool {
	url := "https://www.dmm.co.jp/digital/videoa/-/series/=/sort=date/"
	doc, err := getPage(url)
	if err != nil {
		log.Println(err)
		return false
	}
	storeSeries(doc)

	pageC := doc.Find(".paginationControl a")
	if pageC.Length() <= 0 {
		time.Sleep(waitTime)
		return false
	}
	a := pageC.Eq(pageC.Length() - 2)
	h, _ := a.Attr("href")
	index := strings.Index(h, "page=")
	if index == -1 {
		return false
	}

	//按最新排序翻页数 若为零则全部
	if page <= 0 {
		page, _ = strconv.Atoi(h[index+5 : len(h)-1])
	}

	for i := 2; i <= page; i++ {
		time.Sleep(waitTime)
		pageUrl := url + "page=" + strconv.Itoa(i)
		doc, err = getPage(pageUrl)
		if err != nil {
			log.Println(err)
			continue
		}
		storeSeries(doc)
	}
	log.Println("getSeries END")
	return false
}

func storeSeries(doc *goquery.Document) {
	doc.Find(".tx-work.mg-b12 a").Each(func(i int, li *goquery.Selection) {
		go func() {
			txt, es := li.Attr("href")
			if !es {
				return
			}

			sid := txt[strings.Index(txt, "id=")+3 : len(txt)-1]
			sid8, _ := strconv.Atoi(sid)
			if mDB.IsSeriesExist(sid8) {
				log.Println(sid8, "existed")
				return
			}

			series := li.Text()
			mDB.InsertSeries(sid8, series)
			log.Println(sid8, series)
		}()
	})
}

func PreGetSeriesMovies() {
	list := mDB.GetScanSeries(1200)
	if len(list) <= 0 {
		log.Println("PreGetSeriesMovies END")
		return
	}
	url := "https://www.dmm.co.jp/digital/videoa/-/list/=/article=series/id="

	for _, sid := range list {
		tmpUrl := url + strconv.Itoa(sid)
		time.Sleep(waitTime)
		doc, err := getPage(tmpUrl)
		if Status404 == err {
			mDB.UptSeriesStatus(sid)
		}
		if err != nil {
			log.Println(tmpUrl, err)
			continue
		}
		PreGetMoviesExce(doc)

		mDB.UptSeriesScan(sid, time.Now().Format("2006-01-02 15:04:05"))

		pageBox := doc.Find(".list-boxpagenation a")
		if pageBox.Length() <= 0 {
			log.Println("Only Page 1")
			continue
		}

		var aTag *goquery.Selection
		txt := pageBox.Last().Text()
		if txt == "次へ" {
			aTag = pageBox.Eq(pageBox.Length() - 2)
		} else {
			aTag = pageBox.Last()
		}
		h, _ := aTag.Attr("href")
		index := strings.Index(h, "page=")
		if index == -1 {
			continue
		}
		page, _ := strconv.Atoi(h[index+5 : len(h)-1])
		log.Println(page)

		for i := 2; i <= page; i++ {
			time.Sleep(waitTime)
			doc, err := getPage(tmpUrl + "/page=" + strconv.Itoa(i))
			if err != nil {
				log.Println(tmpUrl, err)
				continue
			}
			PreGetMoviesExce(doc)
		}
	}
	//PreGetSeriesMovies(before)
}

//--------------getPage------------//
func newHttpClient() {
	urli := netUrl.URL{}
	urlproxy, _ := urli.Parse("http://127.0.0.1:1080")
	jar := new(Jar)
	httpClient = &http.Client{Jar: jar, Timeout: 3 * waitTime, Transport: &http.Transport{
		Proxy: http.ProxyURL(urlproxy),
	}}
}

type Jar struct {
	cookies []*http.Cookie
}

func (jar *Jar) SetCookies(u *netUrl.URL, cookies []*http.Cookie) {
	jar.cookies = append(jar.cookies, cookies...)
}
func (jar *Jar) Cookies(u *netUrl.URL) []*http.Cookie {
	return jar.cookies
}

var (
	StatusErr = errors.New("res.Status")
	Status404 = errors.New("404")
)

func getPage(url string) (doc *goquery.Document, err error) {
	log.Println("getPage", url)

	url = "https://www.dmm.co.jp/age_check/=/declared=yes/?rurl=" + netUrl.QueryEscape(url)
	newHttpClient()
	req, _ := http.NewRequest("GET", url, strings.NewReader(""))
	req.Header.Set("user-agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36")

	var res *http.Response
	for i := 0; i < 1; i++ {
		res, err = httpClient.Do(req)
		if err != nil {
			log.Println(err)
			continue
		}

		if res.StatusCode == 404 {
			log.Println("status code error: ", res.StatusCode, res.Status)
			res.Body.Close()
			return nil, Status404
		}

		if res.StatusCode != 200 {
			log.Println("status code error: ", res.StatusCode, res.Status)
			err = errors.New(res.Status)
			res.Body.Close()
			newHttpClient()
			continue
		}
		break
	}

	// Load the HTML document
	if res == nil || res.StatusCode != 200 {
		log.Println("StatusCode:", res)
		err = StatusErr
		return
	}
	defer res.Body.Close()
	doc, err = goquery.NewDocumentFromReader(res.Body)
	if err != nil {
		log.Println(err)
	}
	return
}
